{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d9d8922",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "import time\n",
    "import torch\n",
    "from PIL import Image\n",
    "from tqdm import tqdm\n",
    "from torch.utils.data import Dataset\n",
    "from torchvision import transforms\n",
    "from collections import Counter\n",
    "\n",
    "# 检查关键文件是否存在\n",
    "def verify_dataset(root=\"E:/codes/project/datas/VeRi-776\"):\n",
    "    required_files = [\n",
    "        \"name_train.txt\", \"name_test.txt\", \"name_query.txt\",\n",
    "        \"camera_ID.txt\", \"image_train\", \"image_test\", \"image_query\"\n",
    "    ]\n",
    "    \n",
    "    missing = []\n",
    "    for f in required_files:\n",
    "        path = Path(root) / f\n",
    "        if not path.exists():\n",
    "            missing.append(str(path))\n",
    "    \n",
    "    if missing:\n",
    "        print(\"以下文件/目录缺失：\")\n",
    "        for m in missing:\n",
    "            print(f\"  - {m}\")\n",
    "    else:\n",
    "        print(\"数据集结构完整！\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class Veri776Dataset(Dataset):   #功能：标准VeRi-776数据集加载器  预处理：包含图像尺寸调整、标准化等transform操作\n",
    "    def __init__(self, mode=\"train\", transform=None, root_dir=None, cache_images=False):\n",
    "        super().__init__()\n",
    "\n",
    "        start_time = time.time()\n",
    "\n",
    "\n",
    "        # ===== 修改位置：添加原始字符串标记 =====\n",
    "        # 设置数据集根目录\n",
    "        if root_dir:\n",
    "            self.root_dir = root_dir\n",
    "        else:\n",
    "            current_dir = os.path.dirname(os.path.abspath(__file__))\n",
    "            self.root_dir = os.path.abspath(\n",
    "                os.path.join(current_dir, r\"E:\\codes\\project\\datas\\VeRi-776\")\n",
    "            )\n",
    "        # ===== 修改结束 =====\n",
    "\n",
    "        # 验证路径有效性\n",
    "        if not os.path.exists(self.root_dir):\n",
    "            raise FileNotFoundError(f\"数据集路径不存在: {self.root_dir}\")\n",
    "\n",
    "        self.mode = mode\n",
    "        self.transform = transform\n",
    "        self.cache_images = cache_images  # 保留参数，但实际不使用缓存\n",
    "        self.cached_images = None  # 不初始化缓存\n",
    "        # 初始化存储容器\n",
    "        self.image_paths = []\n",
    "        self.vehicle_ids = []\n",
    "        self.camera_ids = []  # 新增摄像头ID列表存储\n",
    "        \n",
    "        \n",
    "        # 模式验证\n",
    "        valid_modes = [\"train\", \"test\", \"query\"]\n",
    "        if mode not in valid_modes:\n",
    "            raise ValueError(f\"Invalid mode {mode}, expected {valid_modes}\")\n",
    "        \n",
    "        # 文件映射\n",
    "        name_file_map = {\n",
    "            \"train\": \"name_train.txt\",\n",
    "            \"test\": \"name_test.txt\", \n",
    "            \"query\": \"name_query.txt\"\n",
    "        }\n",
    "        image_dir_map = {\n",
    "            \"train\": \"image_train\",\n",
    "            \"test\": \"image_test\",\n",
    "            \"query\": \"image_query\"\n",
    "        }\n",
    "        \n",
    "        # 读取文件名列表\n",
    "        name_file = os.path.join(self.root_dir, name_file_map[mode])\n",
    "        if not os.path.exists(name_file):\n",
    "            raise FileNotFoundError(f\"{name_file} 不存在!\")\n",
    "        with open(name_file, \"r\") as f:\n",
    "            file_names = [line.strip() for line in f.readlines()]\n",
    "        print(f\"共加载 {len(file_names)} 个文件名，耗时 {time.time()-start_time:.2f}s\")\n",
    "        \n",
    "\n",
    "                 \n",
    "                    \n",
    "\n",
    "        # 构建数据列表（带进度条）\n",
    "        image_dir = os.path.join(self.root_dir, image_dir_map[mode])\n",
    "        self.data = []  # 存储 (路径, 车辆ID, 相机ID) 三元组\n",
    "\n",
    "        for file_name in tqdm(file_names, desc=f\"处理 {mode} 数据\"):\n",
    "            # 解析车辆ID\n",
    "            try:\n",
    "                vehicle_id = int(file_name.split(\"_\")[0])\n",
    "            except:\n",
    "                print(f\"无法解析文件名: {file_name}\")\n",
    "                continue\n",
    "\n",
    "\n",
    "\n",
    "            # +++ 新增摄像头ID解析部分 +++\n",
    "            try:\n",
    "                cam_part = file_name.split(\"_\")[1]    # 例如提取c002\n",
    "                cam_id = int(cam_part[1:])            # 转换为整数2\n",
    "            except Exception as e:\n",
    "                print(f\"无法解析摄像头ID: {file_name}, 错误: {str(e)}\")\n",
    "                cam_id = 0\n",
    "            \n",
    "            \n",
    "\n",
    "\n",
    "\n",
    "            # 检查文件存在性\n",
    "            img_path = os.path.join(image_dir, file_name)\n",
    "            if not os.path.exists(img_path):\n",
    "                print(f\"文件缺失: {img_path}\")\n",
    "                continue\n",
    "                \n",
    "            # 存储三元组：(路径, 车辆ID, 相机ID)\n",
    "            self.data.append((img_path, vehicle_id, cam_id))\n",
    "            self.vehicle_ids.append(vehicle_id)\n",
    "            self.camera_ids.append(cam_id)\n",
    "\n",
    "            \n",
    "\n",
    "\n",
    "         # ID映射（连续化处理）\n",
    "        self.unique_ids = sorted(set(self.vehicle_ids))\n",
    "        self.id_to_class = {v: k for k, v in enumerate(self.unique_ids)}\n",
    "\n",
    "\n",
    "         # 检查ID连续性\n",
    "        if mode == \"train\":\n",
    "            class_ids = [self.id_to_class[vid] for vid in self.vehicle_ids]\n",
    "            min_class = min(class_ids)\n",
    "            max_class = max(class_ids)\n",
    "            if min_class != 0 or max_class != len(self.unique_ids) - 1:\n",
    "                print(f\"警告: 训练集类别ID不连续，范围: [{min_class}, {max_class}]\")\n",
    "\n",
    "\n",
    "\n",
    "     \n",
    "   \n",
    "        \n",
    "        \n",
    "\n",
    "\n",
    "        \n",
    "\n",
    "\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        img_path, vehicle_id, cam_id = self.data[idx]\n",
    "        img = Image.open(img_path).convert(\"RGB\")  # 动态加载图像\n",
    "\n",
    "\n",
    "\n",
    "        if self.transform:\n",
    "            img = self.transform(img)\n",
    "        return {\n",
    "            \"image\": img,\n",
    "            \"vehicle_id\": vehicle_id,\n",
    "            \"class_id\": self.id_to_class[vehicle_id],\n",
    "            \"camera_id\": cam_id,\n",
    "            \"path\": img_path\n",
    "            \n",
    "        }\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\n",
    "\n",
    "    verify_dataset()\n",
    "    \n",
    "\n",
    "\n",
    "    # 极简模式测试\n",
    "    train_transform = transforms.Compose([\n",
    "        transforms.Resize(288),  # 先调整为稍大的尺寸\n",
    "    transforms.CenterCrop(256),  # 从中心裁剪出的区域\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    "\n",
    "\n",
    "    # 测试数据集加载\n",
    "    for mode in [\"train\", \"query\", \"test\"]:\n",
    "        print(f\"\\n=== 测试 {mode} 数据集 ===\")\n",
    "        dataset = Veri776Dataset(mode=mode, transform=train_transform)\n",
    "        \n",
    "        # 检查相机ID分布\n",
    "        camera_counts = Counter(dataset.camera_ids)\n",
    "        print(f\"摄像头ID分布: {camera_counts}\")\n",
    "\n",
    "\n",
    "\n",
    "  \n",
    "\n",
    "     \n",
    "\n",
    "    print(\"=== 测试开始 ===\")\n",
    "    try:\n",
    "        \n",
    "        loader = torch.utils.data.DataLoader(\n",
    "            dataset,\n",
    "            batch_size=24,\n",
    "            num_workers=4,\n",
    "            shuffle=True\n",
    "        )\n",
    "        batch = next(iter(loader))\n",
    "        print(\"数据加载测试成功！\")\n",
    "        print(\"图像尺寸:\", batch[\"image\"].shape)\n",
    "        # 验证ID连续性\n",
    "        print(\"\\n=== ID连续性验证 ===\")\n",
    "        print(\"车辆ID范围:\", min(batch[\"vehicle_id\"]), max(batch[\"vehicle_id\"]))\n",
    "        print(\"类别ID范围:\", min(batch[\"class_id\"]), max(batch[\"class_id\"]))\n",
    "        print(\"相机ID范围:\", min(batch[\"camera_id\"]), max(batch[\"camera_id\"]))\n",
    "\n",
    "        \n",
    "        \n",
    "    except Exception as e:\n",
    "        print(f\"错误发生: {str(e)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f2df129",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "import time\n",
    "import torch\n",
    "from PIL import Image\n",
    "from tqdm import tqdm\n",
    "from torch.utils.data import Dataset\n",
    "from torchvision import transforms\n",
    "from collections import Counter\n",
    "\n",
    "# 检查关键文件是否存在\n",
    "def verify_dataset(root=\"E:/codes/project/datas/VeRi-776\"):\n",
    "    required_files = [\n",
    "        \"name_train.txt\", \"name_test.txt\", \"name_query.txt\",\n",
    "        \"camera_ID.txt\", \"image_train\", \"image_test\", \"image_query\"\n",
    "    ]\n",
    "    \n",
    "    missing = []\n",
    "    for f in required_files:\n",
    "        path = Path(root) / f\n",
    "        if not path.exists():\n",
    "            missing.append(str(path))\n",
    "    \n",
    "    if missing:\n",
    "        print(\"以下文件/目录缺失：\")\n",
    "        for m in missing:\n",
    "            print(f\"  - {m}\")\n",
    "    else:\n",
    "        print(\"数据集结构完整！\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class Veri776Dataset(Dataset):   #功能：标准VeRi-776数据集加载器  预处理：包含图像尺寸调整、标准化等transform操作\n",
    "    def __init__(self, mode=\"train\", transform=None, root_dir=None, cache_images=False, id_to_class=None):\n",
    "        super().__init__()\n",
    "\n",
    "        start_time = time.time()\n",
    "\n",
    "\n",
    "     \n",
    "        # 设置数据集根目录\n",
    "        if root_dir:\n",
    "            self.root_dir = root_dir\n",
    "        else:\n",
    "            current_dir = os.path.dirname(os.path.abspath(__file__))\n",
    "            self.root_dir = os.path.abspath(\n",
    "                os.path.join(current_dir, r\"E:\\codes\\project\\datas\\VeRi-776\")\n",
    "            )\n",
    "       \n",
    "\n",
    "        # 验证路径有效性\n",
    "        if not os.path.exists(self.root_dir):\n",
    "            raise FileNotFoundError(f\"数据集路径不存在: {self.root_dir}\")\n",
    "\n",
    "        self.mode = mode\n",
    "        self.transform = transform\n",
    "        self.cache_images = cache_images  # 保留参数，但实际不使用缓存\n",
    "        self.cached_images = None  # 不初始化缓存\n",
    "        # 初始化存储容器\n",
    "        self.image_paths = []\n",
    "        self.vehicle_ids = []\n",
    "        self.camera_ids = []  # 摄像头ID列表存储\n",
    "        \n",
    "        \n",
    "        # 模式验证\n",
    "        valid_modes = [\"train\", \"test\", \"query\"]\n",
    "        if mode not in valid_modes:\n",
    "            raise ValueError(f\"Invalid mode {mode}, expected {valid_modes}\")\n",
    "        \n",
    "        # 文件映射\n",
    "        name_file_map = {\n",
    "            \"train\": \"name_train.txt\",\n",
    "            \"test\": \"name_test.txt\", \n",
    "            \"query\": \"name_query.txt\"\n",
    "        }\n",
    "        image_dir_map = {\n",
    "            \"train\": \"image_train\",\n",
    "            \"test\": \"image_test\",\n",
    "            \"query\": \"image_query\"\n",
    "        }\n",
    "        \n",
    "        # 读取文件名列表\n",
    "        name_file = os.path.join(self.root_dir, name_file_map[mode])\n",
    "        if not os.path.exists(name_file):\n",
    "            raise FileNotFoundError(f\"{name_file} 不存在!\")\n",
    "        with open(name_file, \"r\") as f:\n",
    "            file_names = [line.strip() for line in f.readlines()]\n",
    "        print(f\"共加载 {len(file_names)} 个文件名，耗时 {time.time()-start_time:.2f}s\")\n",
    "        \n",
    "\n",
    "                 \n",
    "                    \n",
    "\n",
    "        # 构建数据列表（带进度条）\n",
    "        image_dir = os.path.join(self.root_dir, image_dir_map[mode])\n",
    "        self.data = []  # 存储 (路径, 车辆ID, 相机ID) 三元组\n",
    "\n",
    "        for file_name in tqdm(file_names, desc=f\"处理 {mode} 数据\"):\n",
    "            # 解析车辆ID\n",
    "            try:\n",
    "                vehicle_id = int(file_name.split(\"_\")[0])\n",
    "            except:\n",
    "                print(f\"无法解析文件名: {file_name}\")\n",
    "                continue\n",
    "\n",
    "\n",
    "\n",
    "            # +++ 摄像头ID解析部分 +++\n",
    "            try:\n",
    "                cam_part = file_name.split(\"_\")[1]    # 例如提取c002\n",
    "                cam_id = int(cam_part[1:])            # 转换为整数2\n",
    "            except Exception as e:\n",
    "                print(f\"无法解析摄像头ID: {file_name}, 错误: {str(e)}\")\n",
    "                cam_id = 0\n",
    "            \n",
    "            \n",
    "\n",
    "\n",
    "\n",
    "            # 检查文件存在性\n",
    "            img_path = os.path.join(image_dir, file_name)\n",
    "            if not os.path.exists(img_path):\n",
    "                print(f\"文件缺失: {img_path}\")\n",
    "                continue\n",
    "                \n",
    "            # 存储三元组：(路径, 车辆ID, 相机ID)\n",
    "            self.data.append((img_path, vehicle_id, cam_id))\n",
    "            self.vehicle_ids.append(vehicle_id)\n",
    "            self.camera_ids.append(cam_id)\n",
    "\n",
    "            # 新增：用于存储未知车辆ID的集合\n",
    "        self.unknown_vehicle_ids = set()\n",
    "\n",
    "\n",
    "         # ID映射（连续化处理）\n",
    "        if id_to_class is not None:\n",
    "            self.id_to_class = id_to_class\n",
    "            # 生成unique_ids（即使使用全局映射）\n",
    "            self.unique_ids = sorted(set(self.vehicle_ids))\n",
    "             # 收集未知车辆ID\n",
    "            for vid in self.vehicle_ids:\n",
    "                if vid not in self.id_to_class:\n",
    "                    self.unknown_vehicle_ids.add(vid)\n",
    "        else:\n",
    "            # 仅训练集生成全局映射\n",
    "            self.unique_ids = sorted(set(self.vehicle_ids))\n",
    "            self.id_to_class = {v: k for k, v in enumerate(self.unique_ids)}\n",
    "\n",
    "\n",
    "         # 检查ID连续性\n",
    "        if mode == \"train\":\n",
    "            class_ids = [self.id_to_class[vid] for vid in self.vehicle_ids]\n",
    "            min_class = min(class_ids)\n",
    "            max_class = max(class_ids)\n",
    "            if min_class != 0 or max_class != len(self.unique_ids) - 1:\n",
    "                print(f\"警告: 训练集类别ID不连续，范围: [{min_class}, {max_class}]\")\n",
    "\n",
    "\n",
    "                # 在初始化最后统一输出未知车辆统计信息\n",
    "        if self.unknown_vehicle_ids and mode != \"train\":\n",
    "            print(f\"警告: {mode}集发现 {len(self.unknown_vehicle_ids)} 个未知车辆ID，ID范围: {min(self.unknown_vehicle_ids)}-{max(self.unknown_vehicle_ids)}\")\n",
    "\n",
    "\n",
    "\n",
    "     \n",
    "   \n",
    "        \n",
    "        \n",
    "\n",
    "\n",
    "        \n",
    "\n",
    "\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        img_path, vehicle_id, cam_id = self.data[idx]\n",
    "        img = Image.open(img_path).convert(\"RGB\")  # 动态加载图像\n",
    "\n",
    "\n",
    "\n",
    "        if self.transform:\n",
    "            img = self.transform(img)\n",
    "        \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        return {\n",
    "            \"image\": img,\n",
    "            \"vehicle_id\": vehicle_id,\n",
    "            \"class_id\": self.id_to_class[vehicle_id],\n",
    "            \"camera_id\": cam_id,\n",
    "            \"path\": img_path\n",
    "            \n",
    "        }\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\n",
    "\n",
    "    verify_dataset()\n",
    "    \n",
    "\n",
    "\n",
    "    # 极简模式测试\n",
    "    train_transform = transforms.Compose([\n",
    "        transforms.Resize(256),  # 先调整为稍大的尺寸\n",
    "    transforms.CenterCrop(224),  # 从中心裁剪出的区域\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    "     # 加载训练集，生成全局ID映射\n",
    "    train_dataset = Veri776Dataset(mode=\"train\", transform=train_transform)\n",
    "    global_id_to_class = train_dataset.id_to_class  # 保存训练集的全局映射\n",
    "\n",
    "     # 加载测试集和查询集，复用训练集的映射\n",
    "    datasets = {}\n",
    "    # 测试数据集加载\n",
    "    for mode in [\"train\", \"query\", \"test\"]:\n",
    "        print(f\"\\n=== 测试 {mode} 数据集 ===\")\n",
    "        dataset = Veri776Dataset(mode=mode, \n",
    "                                 transform=train_transform ,\n",
    "                                 id_to_class=global_id_to_class  # 传入全局映射\n",
    "                                 \n",
    "                                 )\n",
    "        datasets[mode] = dataset\n",
    "        # 检查相机ID分布\n",
    "        camera_counts = Counter(dataset.camera_ids)\n",
    "        print(f\"摄像头ID分布: {camera_counts}\")\n",
    "\n",
    "\n",
    "\n",
    "  \n",
    "\n",
    "     \n",
    "\n",
    "    print(\"=== 测试开始 ===\")\n",
    "    try:\n",
    "        \n",
    "        loader = torch.utils.data.DataLoader(\n",
    "            datasets[\"train\"],  # 指定使用训练集\n",
    "            batch_size=16,\n",
    "            num_workers=4,\n",
    "            shuffle=True\n",
    "        )\n",
    "        batch = next(iter(loader))\n",
    "        print(\"数据加载测试成功！\")\n",
    "        print(\"图像尺寸:\", batch[\"image\"].shape)\n",
    "        # 验证ID连续性\n",
    "        print(\"\\n=== ID连续性验证 ===\")\n",
    "        print(\"训练集车辆ID范围:\", min(train_dataset.vehicle_ids), max(train_dataset.vehicle_ids))\n",
    "        print(\"训练集类别ID范围:\", 0, len(train_dataset.unique_ids) - 1)  # 应连续0~N-1\n",
    "        print(\"训练集相机ID范围:\", min(train_dataset.camera_ids), max(train_dataset.camera_ids))\n",
    "\n",
    "        \n",
    "        \n",
    "    except Exception as e:\n",
    "        print(f\"错误发生: {str(e)}\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#警告: query集发现 200 个未知车辆ID，ID范围: 2-776\n",
    "#警告: test集发现 200 个未知车辆ID，ID范围: 2-776\n",
    "#VeRi-776 数据集的训练集、查询集、测试集是按摄像头划分的（而非按车辆 ID），因此：\n",
    "#训练集包含部分摄像头的车辆图像；\n",
    "#查询集和测试集包含其他摄像头的车辆图像，其中许多车辆未在训练集中出现。\n",
    "#代码强制要求测试 / 查询集的车辆 ID 必须存在于训练集的映射中，不存在时输出警告。\n",
    "# 这些警告是 Veri776 数据集开放式测试的正常现象，不影响模型训练和评估结果\n",
    "\n",
    "#开放式测试：\n",
    "#测试集包含训练集未出现的车辆，更符合实际场景（如监控场景新增车辆），Veri776 采用此设置。\n",
    "#模型需要学习车辆特征的泛化能力，而非死记硬背训练集车辆。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch2.0",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
